from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA
from scipy.stats import pearsonr
import jieba
import pandas as pd

"""
pip install scikit-learn
pip install jieba
pip install pandas
pip install scipy
"""


def datasets_demo():
    iris = load_iris()
    print("鸢尾花数据集:\n", iris)
    print("查看数据集的描述:\n", iris["DESCR"])
    print("查看特征值的名字：\n", iris.feature_names)
    print("查看特征值：\n", iris.data)

    # 在这段代码中，random_state=22是设置的随机种子。因为随机种子是固定的，所以每次运行程序时都会得到相同的训练集和测试集。这对于代码的可重复性和调试非常有用。
    x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=22)
    print("训练集的特征值：\n", x_train, x_train.shape)
    return None


def dict_demo():
    """
    字典特征抽取
    :return:
    """
    data = [{'city': '北京', 'temperature': 100}, {'city': '上海', 'temperature': 60}, {'city': '深圳', 'temperature': 30}]
    # 1.实例化一个转换器类
    transfer = DictVectorizer(sparse=False)
    # 2.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new \n", data_new)
    print("特征名字：\n", transfer.get_feature_names_out())


def count_demo():
    """
    文本特征抽取,统计每个样本特征词出现的个数
    :return:
    """
    data = ["life is short,i like like python", "life is too long,i dislike python"]
    # 1.实例化一个转换器类
    #transfer = CountVectorizer()
    # stop_words停用词
    transfer = CountVectorizer(stop_words=["is","too"])
    # 2.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new \n", data_new.toarray())
    print("特征名字：\n", transfer.get_feature_names_out())


def count_chinese_demo():
    """
    中文文本特征抽取
    :return:
    """
    data = ["我 爱 北京 天安门", "天安门 上 太阳 升"]
    # 1.实例化一个转换器类
    transfer = CountVectorizer()
    # 2.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new \n", data_new.toarray())
    print("特征名字：\n", transfer.get_feature_names_out())

def count_chinese_demo2():
    """
    中文文本特征抽取，自动分词
    :return:
    """
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_new = []
    for sent in data:
        data_new.append(cut_word(sent))

    transfer = CountVectorizer()
    # 2.调用fit_transform
    data_final = transfer.fit_transform(data_new)
    print("data_new \n", data_final.toarray())
    print("特征名字：\n", transfer.get_feature_names_out())

def cut_word(text):
    """
    进行中文分词
    :param text:
    :return:
    """
    a = " ".join(list(jieba.cut(text)))
    return a


def tfidf_demo():
    """
    用TF-IDF的方法进行文本特征的抽取
    :return:
    """
    data = ["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。",
            "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去。",
            "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系。"]
    data_new = []
    for sent in data:
        data_new.append(cut_word(sent))

    transfer = TfidfVectorizer()
    # 2.调用fit_transform
    data_final = transfer.fit_transform(data_new)
    print("data_new \n", data_final.toarray())
    print("特征名字：\n", transfer.get_feature_names_out())

def minmax_demo():
    """
    归一化
    :return:
    """
    # 1.获取数据
    data = pd.read_csv("dating.txt")
    # 获取前三列
    data = data.iloc[:, :3]
    #print("data:\n", data)
    # 2.实例化一个转换器类
    transfer = MinMaxScaler()
    # 3.调用fit_transform
    data_new= transfer.fit_transform(data)
    print("data_new:\n", data_new)


def variance_demo():
    """
    过滤低方差特征
    :return:
    """
    # 1.获取数据
    data = pd.read_csv("factor_returns.csv")
    data = data.iloc[:,1:-2]
    print("data:\n",data)
    # 2.实例化一个转换器,threshold设置阈值
    transfer = VarianceThreshold(threshold=10)
    # 3.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new:\n", data_new,data_new.shape)

    # 计算两个变量之间的相关系数
    r = pearsonr(data["pe_ratio"],data["pb_ratio"])
    print("相关系数：\n",r)
    r2 = pearsonr(data["revenue"],data["total_expense"])
    print("revenue与total_expense之间的相关性：\n",r2)

def pca_demo():
    """
    PCA降维
    :return:
    """
    data = [[2,8,4,5],[6,3,0,8],[5,4,9,1]]
    # 1.实例化一个转换器类
    transfer = PCA(n_components=2)
    # 2.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new:\n",data_new)


def instacart_pca_demo():
    """
    探究用户对物品类别的喜好细分降维
    :return:
    """
    # 1.获取数据
    order_products = pd.read_csv("./instacart/order_products__prior.csv")
    products = pd.read_csv("./instacart/products.csv")
    orders = pd.read_csv("./instacart/orders.csv")
    aisles = pd.read_csv("./instacart/aisles.csv")

    # 合并aisles和products
    tb1 = pd.merge(aisles,products, on=["aisle_id", "aisle_id"])
    tb2 = pd.merge(tb1, order_products, on=["product_id","product_id"])
    tb3 = pd.merge(tb2,orders, on=["order_id","order_id"])

    # 找到user_id和aisle之间的关系
    table = pd.crosstab(tb3["user_id"],tb3["aisle"])
    data = table[:10000]
    # 1.实例化一个转换器类
    transfer = PCA(n_components=0.95)
    # 2.调用fit_transform
    data_new = transfer.fit_transform(data)
    print("data_new:\n", data_new)
if __name__ == "__main__":
    # datasets_demo()
    #dict_demo()
    #count_demo()
    #count_chinese_demo()
    #count_chinese_demo2()
    #tfidf_demo()
    #minmax_demo()
    #variance_demo()
    #pca_demo()
    instacart_pca_demo()
